import requests
from requests.exceptions import HTTPError,RequestException
import pymongo
import math,json

MONGO_URL='localhost'
MONGO_DB='llagou'
MONGO_TABLE='数据分析'

CITY = '深圳'
KEY_WORD='数据分析师'
BASE_URL = 'https://www.lagou.com/jobs/positionAjax.json'
headers = {
    'Accept':'application/json, text/javascript, */*; q=0.01',
    'Accept-Encoding':'gzip, deflate, br',
    'Accept-Language':'zh-CN,zh;q=0.9',
    'Connection':'keep-alive',
    'Content-Length':'64',
    'Cookie':'_ga=GA1.2.257516341.1522050452; user_trace_token=20180326154732-f0f5196c-30c9-11e8-9f49-525400f775ce; LGUID=20180326154732-f0f520df-30c9-11e8-9f49-525400f775ce; LG_LOGIN_USER_ID=60032b76bd32a5a6a596559eade6d0052f15fc2ab9779cc8; index_location_city=%E6%B7%B1%E5%9C%B3; JSESSIONID=ABAAABAAAIAACBI1A73582AEDC2FDC8335E968502A9F5D2; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1527150390,1528277552,1528789503,1529461642; _gid=GA1.2.1084200828.1529461642; LGSID=20180620153431-5f3689a6-745c-11e8-aa10-525400f775ce; TG-TRACK-CODE=search_code; LGRID=20180620160449-9a8ba121-7460-11e8-aa14-525400f775ce; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1529481888; SEARCH_ID=efa038bbf91d47488b5c710088432205',
    'Host':'www.lagou.com',
    'Origin':'https://www.lagou.com',
    'Referer':'https://www.lagou.com/jobs/list_%E6%95%B0%E6%8D%AE%E5%88%86%E6%9E%90%E5%B8%88?px=default&city=%E6%B7%B1%E5%9C%B3',
    'Content-Type':'application/x-www-form-urlencoded; charset=UTF-8',
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}
page = 0

client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_DB]
mySet = db[MONGO_TABLE]


def get_one_page(p=1):
    global page
    data = {
        'first': 'true',
        'pn': p,
        'kd': KEY_WORD,
    }
    params = {
        'city': CITY,
        'needAddtionalResult': 'false',
    }
    try:
        response = requests.post(url=BASE_URL,params=params,data=data,headers=headers)
        if response.status_code == requests.codes.ok:
            html = response.json()
            if 1 == p:
                pagesize = html['content']['pageSize']
                totalcount = html['content']['positionResult']['totalCount']
                page = math.ceil(totalcount/pagesize)
            result = html['content']['positionResult']['result']
            for i in range(len(result)):
                save_database(result[i])
            return html
        else:
            print("status 错误",response.status_code)
            return None
    except HTTPError:
        print('Http Error')
    except RequestException:
        print('Error')

def save_database(data):
    if mySet.insert(data):
        print("插入成功",data)
    else:
        print("插入失败",data)

def main():
    get_one_page(1)
    for p in range(2,page+1):
        get_one_page(p)

if __name__ == '__main__':
    main()
    print("end")